import scrapy

from scrapy.selector import Selector
from douban.items import DoubanItem


class DoubanXpathSpider(scrapy.Spider):
    name = 'douban-xpath'

    start_urls = ['https://movie.douban.com/top250']

    def parse(self, response):
        # print('body',response.body)
        selector = Selector(response)
        items = DoubanItem()

        # print(selector)
        movies = selector.xpath('//div[@class="info"]')
        for movie in movies:
            movie_name = movie.xpath('div[@class="hd"]/a/span/text()').extract_first()
            movie_star = movie.xpath('div[@class="bd"]/div[@class="star"]/span[@class="rating_num"]/text()').extract_first()
            movie_quote = movie.xpath('div[@class="bd"]/p[@class="quote"]/span[@class="inq"]/text()').extract_first()
            movie_type = movie.xpath('normalize-space(div[@class="bd"]/p/text())').extract_first()
            comment_num = movie.xpath('div[@class="bd"]/div[@class="star"]/span[last()]/text()').extract_first()

            items['movie_name'] = movie_name
            items['movie_star'] = movie_star
            items['comment_num'] = comment_num
            items['movie_quote'] = movie_quote
            items['movie_type'] = movie_type

            yield items

        page = selector.xpath("//div[@class='paginator']/span[@class='thispage']/text()").extract_first()
        url = 'https://movie.douban.com/top250?start='+ str(int(page) * 25) +'&filter='
        last = selector.xpath("//div[@class='paginator']/a[last()]/@href").extract_first()

        print(page)
       
        if int(page) != 10:
            yield scrapy.Request(response.urljoin(url))